import numpy as np
import matplotlib.pyplot as plt
from matplotlib import gridspec
import seaborn as sns
import glob
import os
import re
import pandas as pd
import tarfile
from sklearn.decomposition import PCA
import P4J

%matplotlib inline
plt.rcParams['figure.figsize'] = (9, 6)
sns.set(style="whitegrid", color_codes=True, context="poster")

mainpath = '/Users/jorgetil/Astro/HITS'

/Users/jorgetil/miniconda/lib/python2.7/site-packages/matplotlib/__init__.py:872: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.
  warnings.warn(self.msg_depr % (key, alt_key))

def give_me_lc(field, CCD, X, Y, extract=False):
    year = field[:-3]
    try:
        tar = tarfile.open("%s/lightcurves/%s/%s/%s/%s_%s_gr_LC_50.tar.gz" 
                           % (mainpath, year, field, CCD, field, CCD))
        fil = tar.extractfile('%s_%s_%s_%s_g.dat' % (field, CCD, X, Y))
        if extract:
            tar.extract('%s_%s_%s_%s_g.dat' % (field, CCD, X, Y),
                    path='/Users/jorgetil/Astro/HITS/lightcurves/samples/.')
    except:
        print 'No tar file or element in tar file'
        return None
    
    time, mag, err = [], [], []
    for line in fil:
        if line[0] == '#': continue
        values = line.split()
        time.append(float(values[1]))
        mag.append(float(values[2]))
        err.append(float(values[3]))
    time = np.asarray(time)
    mag= np.asarray(mag)
    err = np.asarray(err)
    
    try:
        fil = tar.extractfile('%s_%s_%s_%s_r.dat' % (field, CCD, X, Y))
        #tar.extract('%s_%s_%s_%s_r.dat' % (field, CCD, X, Y)
        #                , path='/Users/jorgetil/Downloads/.')
        time2, mag2, err2 = [], [], []
        for line in fil:
            if line[0] == '#': continue
            values = line.split()
            time2.append(float(values[1]))
            mag2.append(float(values[2]))
            err2.append(float(values[3]))
        time2 = np.asarray(time2)
        mag2 = np.asarray(mag2)
        err2 = np.asarray(err2)
        return time, mag, err, time2, mag2, err2
    except:
        print 'No lightcurve for other filter'
        return time, mag, err, None, None, None

# load feature table into DF
#table_file = '%s/tables/Blind14A_tables+feat_pl_var_type_spCL_spClass.csv' % (mainpath)
table_file = '%s/tables/Blind15A_label_sample.csv' % (mainpath)
table_15 = pd.read_csv(table_file)
table_15.set_index('ID', inplace=True)

table_15.info()

<class 'pandas.core.frame.DataFrame'>
Index: 2717 entries, Blind15A_04_N1_1366_0146 to Blind15A_47_S27_0978_3515
Data columns (total 85 columns):
DEC                                2717 non-null float64
ELLIPTICITY                        2717 non-null float64
FLAGS                              2717 non-null int64
FLUX_RADIUS                        2717 non-null float64
FWHM                               2717 non-null float64
KRON_RADIUS                        2717 non-null float64
Median_err_g                       2717 non-null float64
Median_err_r                       2695 non-null float64
Median_g                           2717 non-null float64
Median_r                           2695 non-null float64
Occu_g                             2717 non-null int64
Occu_r                             2717 non-null int64
RA                                 2717 non-null float64
STD_g                              2717 non-null float64
STD_r                              2695 non-null float64
X                                  2717 non-null float64
Y                                  2717 non-null float64
Amplitude                          2717 non-null float64
AndersonDarling                    2717 non-null float64
Autocor_length                     2717 non-null float64
Beyond1Std                         2717 non-null float64
CAR_mean                           2717 non-null float64
CAR_sigma                          2717 non-null float64
CAR_tau                            2717 non-null float64
Color                              2675 non-null float64
Con                                2717 non-null float64
Eta_e                              2717 non-null float64
FluxPercentileRatioMid20           2717 non-null float64
FluxPercentileRatioMid35           2717 non-null float64
FluxPercentileRatioMid50           2717 non-null float64
FluxPercentileRatioMid65           2717 non-null float64
FluxPercentileRatioMid80           2717 non-null float64
Freq1_harmonics_amplitude_0        2717 non-null float64
Freq1_harmonics_amplitude_1        2717 non-null float64
Freq1_harmonics_amplitude_2        2717 non-null float64
Freq1_harmonics_amplitude_3        2717 non-null float64
Freq1_harmonics_rel_phase_0        2717 non-null float64
Freq1_harmonics_rel_phase_1        2717 non-null float64
Freq1_harmonics_rel_phase_2        2717 non-null float64
Freq1_harmonics_rel_phase_3        2717 non-null float64
Freq2_harmonics_amplitude_0        2717 non-null float64
Freq2_harmonics_amplitude_1        2717 non-null float64
Freq2_harmonics_amplitude_2        2717 non-null float64
Freq2_harmonics_amplitude_3        2717 non-null float64
Freq2_harmonics_rel_phase_0        2717 non-null float64
Freq2_harmonics_rel_phase_1        2717 non-null float64
Freq2_harmonics_rel_phase_2        2717 non-null float64
Freq2_harmonics_rel_phase_3        2717 non-null float64
Freq3_harmonics_amplitude_0        2717 non-null float64
Freq3_harmonics_amplitude_1        2717 non-null float64
Freq3_harmonics_amplitude_2        2717 non-null float64
Freq3_harmonics_amplitude_3        2717 non-null float64
Freq3_harmonics_rel_phase_0        2717 non-null float64
Freq3_harmonics_rel_phase_1        2717 non-null float64
Freq3_harmonics_rel_phase_2        2717 non-null float64
Freq3_harmonics_rel_phase_3        2717 non-null float64
Gskew                              2717 non-null float64
LinearTrend                        2717 non-null float64
MaxSlope                           2717 non-null float64
Mean                               2717 non-null float64
Meanvariance                       2717 non-null float64
MedianAbsDev                       2717 non-null float64
MedianBRP                          2717 non-null float64
PairSlopeTrend                     2717 non-null float64
PercentAmplitude                   2717 non-null float64
PercentDifferenceFluxPercentile    2717 non-null float64
PeriodLS                           2717 non-null float64
Period_fit                         2717 non-null float64
Psi_CS                             2717 non-null float64
Psi_eta                            2717 non-null float64
Q31                                2717 non-null float64
Rcs                                2717 non-null float64
Skew                               2717 non-null float64
SlottedA_length                    2717 non-null float64
SmallKurtosis                      2717 non-null float64
Std                                2717 non-null float64
StetsonK                           2717 non-null float64
StetsonK_AC                        2717 non-null float64
WMCC_Period                        2717 non-null float64
WMCC_conf                          2717 non-null float64
spCl                               2580 non-null object
subClass                           1921 non-null object
Var_Type                           2717 non-null object
Var_subType                        20 non-null object
Pred_Var_Type                      0 non-null float64
dtypes: float64(78), int64(3), object(4)
memory usage: 1.8+ MB

# CMD of detections g = sns.jointplot(x="Color", y="Mean", data=table_15, kind="scatter", stat_func=None, size=10, ratio=5, color='b')# distribution of Perdio from LS plt.hist(table_15['MedianAbsDev'], bins=50, color='b', log=True, histtype = 'stepfilled', label='All', alpha=0.7) #plt.hist(table_15.query('Var_Type=="QSO"')['Eta_e'], color='g', log=True, # histtype = 'stepfilled', label='QSO') plt.legend(loc='best') plt.xlabel('MedianAbsDev') plt.ylabel('N') plt.show()# distribution of Perdio from LS plt.hist(table_15['Psi_eta'], bins=50, color='b', log=True, histtype = 'stepfilled', label='All', alpha=0.7) #plt.hist(table_15.query('Var_Type=="QSO"')['Psi_eta'], color='g', log=True, # histtype = 'stepfilled', label='QSO') plt.legend(loc='best') plt.xlabel('Psi_eta') plt.ylabel('N') plt.show()plt.hist(table_15['Meanvariance'], bins=50, color='g', log=True, alpha=0.7, histtype = 'stepfilled') plt.show()# distribution of Period_fit plt.hist(table_15['Std'], bins=50, color='b', log=True, histtype = 'stepfilled', label='All', alpha=0.7) #plt.hist(table_15.query('spCl=="GALAXY"')['Meanvariance'], color='g', log=True, # histtype = 'stepfilled', label='GALAXY') #plt.hist(table_15.query('Var_Type=="RRLYR"')['Meanvariance'], color='r', log=True, # histtype = 'stepfilled', label='RRLYR') plt.legend(loc='best') plt.xlabel('Period_fit') plt.ylabel('N') plt.show()def threshold(a, threshmin=None, threshmax=None): """ Clip array to a given value. Similar to numpy.clip(), except that values less than `threshmin` or greater than `threshmax` are replaced by `newval`, instead of by `threshmin` and `threshmax` respectively. Parameters ---------- a : ndarray Input data threshmin : {None, float}, optional Lower threshold. If None, set to the minimum value. threshmax : {None, float}, optional Upper threshold. If None, set to the maximum value. newval : {0, float}, optional Value outside the thresholds. Returns ------- threshold : ndarray Returns `a`, with values less then `threshmin` and values greater `threshmax` replaced with `newval`. """ a = np.asarray(a).copy() mask = np.zeros(a.shape, dtype=bool) if threshmin is not None: mask |= (a < threshmin) if threshmax is not None: mask |= (a > threshmax) return ~maskperiodic_only = table_15.query('Var_Type == "RRLYR"or Var_Type == "CEP" or Var_Type == "EB"').index ratio = table_15.WMCC_Period[periodic_only]/table_15.PeriodLS[periodic_only] idx_equal = ratio.index[threshold(ratio.values, threshmin=0.9, threshmax=1.1)] idx_non_equal = ratio.index[~threshold(ratio.values, threshmin=0.9, threshmax=1.1)] print periodic_only.shape print table_15.loc[idx_equal].shape print table_15.loc[idx_non_equal].shape table_15_filter = table_15.loc[idx_non_equal] table_15_filter

plt.plot(table_15.PeriodLS, table_15.WMCC_Period, 'b.')
plt.xlim(0,15)
plt.xlim(0,15)
plt.show()

table_15_sub = table_15.loc[['Blind15A_42_S21_1410_0699','Blind15A_48_S10_1986_1958', 'Blind15A_20_S17_1015_1087']]

rrl_ = table_15.query('Var_Type == "RRLYR"')
cep_ = table_15.query('Var_Type == "CEP"')
ebs_ = table_15.query('Var_Type == "EB"')
plt.plot(rrl_.PeriodLS, rrl_.Amplitude, '.b', label='RRLYR')
plt.plot(cep_.PeriodLS, cep_.Amplitude, '.r', label='CEP')
plt.plot(ebs_.PeriodLS, ebs_.Amplitude, '.g', label='EB')
plt.legend(loc='best')
plt.xscale('log')
plt.show()

WCMM_new = []
WCMM_old = []
ls = []
periodogram = True
#for idx, row in table_15.query('WMCC_conf >= 0.98 and PeriodLS >= 1.5').head(50).iterrows():
for idx, row in table_15.query('Var_Type == "CEP"').iterrows():
    print '\r',idx
    label = row['Var_Type']
    print row[['Var_Type','FLUX_RADIUS','ELLIPTICITY','FWHM',
               'FLAGS','Mean','Median_g',
               'Std','MedianAbsDev','Meanvariance','Eta_e',
               'Pred_Var_Type','Prob_Pred','PeriodLS',
               'Period_fit', 'WMCC_Period', 'WMCC_conf']]
    T = float(row['PeriodLS'])
    field, CCD, X, Y = re.findall(
            r'(\w+\d+\w?\_\d\d?)\_(\w\d+?)\_(\d+)\_(\d+)', idx)[0]
    try:
        time, mag, err, time2, mag2, err2 = give_me_lc(field, CCD, X, Y,extract=False)
    except:
        print 'fail during LC read'
        continue
    
    if periodogram:
        M = 1
        WMCC_model = P4J.periodogram(M=M, method='WMCC')
        WMCC_model.fit(time, mag, err)
        freq, per = WMCC_model.grid_search(fmin=1/30.0, fmax=1/.01, fres_coarse=2,
                                            fres_fine=0.05, n_local_max=10)
        fbest = WMCC_model.get_best_frequency()
        WMCC_model.fit_extreme_cdf(n_bootstrap=40, n_frequencies=40)
        falsa_alarm = np.asarray([0.05, 0.01, 0.001])
        per_levels = WMCC_model.get_FAP(falsa_alarm)
        confidence_best_freq = WMCC_model.get_confidence(fbest[1])
        
        print 1/fbest[0]
        
        fig = plt.figure(figsize=(15,14))
        gs = gridspec.GridSpec(3,2)
        ax1 = fig.add_subplot(gs[0,:])
        ax2 = fig.add_subplot(gs[1,0])
        ax3 = fig.add_subplot(gs[1,1])
        ax4 = fig.add_subplot(gs[2,1])
        ax5 = fig.add_subplot(gs[2,0])
        
        # plot of periodogram from WMCC
        ax1.plot(freq, per, 'k-', linewidth=1)
        ax1.axvline(1/T, ls=':', c='c', linewidth=4, alpha=0.9)
        ax1.axvline(2/T, ls=':', c='y', linewidth=4, alpha=0.9)
        # Print confidence bars
        xmin, xmax = ax1.get_xlim()
        for i in range(0, len(falsa_alarm)):
            ax1.axhline(per_levels[i], ls='--', c='b', linewidth=4, alpha=0.5)
            ax1.annotate('%0.3f' % (1.0-falsa_alarm[i]), xy=(xmin+0.01*(xmax-xmin),
                                                       per_levels[i]), fontsize=12)
        # Print max of periodogram
        ymin, ymax = ax1.get_ylim()
        ax1.axvline(fbest[0], ls='--', c='r', linewidth=4, alpha=0.7)
    
        ax1.set_ylim([ymin, ymax])
        ax1.set_xlabel('Frequency [1/MJD]', fontsize=15)
        ax1.set_ylabel('Periodogram', fontsize=15)
        ax1.annotate(' %0.3f' % (confidence_best_freq),
                    xy=(fbest[0], fbest[1]) , fontsize=12)
            
        # LC 
        ax2.errorbar(time, mag, yerr=err, fmt='b*')
        ax2.set_xlabel('time')
        
        # phase LC from LS
        phase1 = np.mod(time, T) / T
        sort_idx1 = np.argsort(phase1)
        PHASE = np.hstack([phase1[sort_idx1], phase1[sort_idx1]+1.])
        MAG = np.hstack([mag[sort_idx1],mag[sort_idx1]])
        ERR = np.hstack([err[sort_idx1],err[sort_idx1]])
        
        ax3.errorbar(PHASE-1, MAG, yerr=ERR, fmt='g*', label='LS')
        ax3.legend(loc='best')
        
        #if label == 'RRLYR' or label == 'EB' or label == 'CEP' or label=='BE':
             
        T_C = float(row['WMCC_Period'])
        t_c = 1/fbest[0]
        ls.append(T)
        WCMM_old.append(T_C)
        WCMM_new.append(t_c)
        print 'WMCC/LS = ', T_C/T
        print 'OLD WMCC period M=3', T_C
        print 'NEW WMCC period M=%i' % (M), t_c
        
        phase_C = np.mod(time, T_C) / T_C
        sort_idx_C = np.argsort(phase_C)
        PHASE_C = np.hstack([phase_C[sort_idx_C], phase_C[sort_idx_C]+1.])
        MAG_C = np.hstack([mag[sort_idx_C],mag[sort_idx_C]])
        ERR_C = np.hstack([err[sort_idx_C],err[sort_idx_C]])
        
        ax4.errorbar(PHASE_C-1, MAG_C, yerr=ERR_C, fmt='b*', alpha=.7, label='WMCC_old')
        ax4.set_xlabel('phase', fontsize=15)
        ax4.legend(loc='best')  
        
        phase_cc = np.mod(time, t_c) / t_c
        sort_idx_cc = np.argsort(phase_cc)
        PHASE_cc = np.hstack([phase_cc[sort_idx_cc], phase_cc[sort_idx_cc]+1.])
        MAG_cc = np.hstack([mag[sort_idx_cc],mag[sort_idx_cc]])
        ERR_cc = np.hstack([err[sort_idx_cc],err[sort_idx_cc]])
        
        ax5.errorbar(PHASE_cc-1, MAG_cc, yerr=ERR_cc, fmt='b*', alpha=.7, label='WMCC_new')
        ax5.set_xlabel('phase', fontsize=15)
        ax5.legend(loc='best')  
            
        ax2.set_ylabel('g', fontsize=15)
        ax2.invert_yaxis()
        ax3.invert_yaxis()
        ax4.invert_yaxis()
        ax5.invert_yaxis()
        plt.show()
        
    else:
        plt.errorbar(time, mag, yerr=err, fmt='b*')
        plt.xlabel('time')
        plt.ylabel('g')
        plt.gca().invert_yaxis()
        plt.show()

Blind15A_27_N3_0379_3173
Var_Type                CEP
FLUX_RADIUS           2.355
ELLIPTICITY       0.0375361
FWHM                   4.24
FLAGS                     0
Mean                16.4351
Median_g            16.4201
Std                0.079917
MedianAbsDev         0.0428
Meanvariance     0.00486259
Eta_e               301.575
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            0.13603
Period_fit        0.0173413
WMCC_Period         0.13603
WMCC_conf          0.999894
Name: Blind15A_27_N3_0379_3173, dtype: object
0.13602952663
WMCC/LS =  1.0
OLD WMCC period M=3 0.13602952663
NEW WMCC period M=1 0.13602952663

Blind15A_27_S6_1014_3281
Var_Type                CEP
FLUX_RADIUS            2.44
ELLIPTICITY       0.0566038
FWHM                   4.38
FLAGS                     0
Mean                18.7493
Median_g            18.7663
Std                 0.17118
MedianAbsDev         0.1641
Meanvariance     0.00912994
Eta_e               154.549
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.335966
Period_fit         0.010955
WMCC_Period        0.335966
WMCC_conf                 1
Name: Blind15A_27_S6_1014_3281, dtype: object
0.335965542282
WMCC/LS =  1.0
OLD WMCC period M=3 0.335965542282
NEW WMCC period M=1 0.335965542282

Blind15A_46_N21_1305_2506
Var_Type               CEP
FLUX_RADIUS          2.364
ELLIPTICITY      0.0448902
FWHM                  4.27
FLAGS                    0
Mean               17.6142
Median_g           17.6526
Std               0.173144
MedianAbsDev        0.1595
Meanvariance     0.0098298
Eta_e              188.711
Pred_Var_Type          NaN
Prob_Pred              NaN
PeriodLS           0.33499
Period_fit       0.0109487
WMCC_Period       0.335065
WMCC_conf                1
Name: Blind15A_46_N21_1305_2506, dtype: object
0.335065055593
WMCC/LS =  1.00022326412
OLD WMCC period M=3 0.335065055593
NEW WMCC period M=1 0.335065055593

Blind15A_16_N10_0158_3879
Var_Type                CEP
FLUX_RADIUS           2.259
ELLIPTICITY        0.101527
FWHM                   4.45
FLAGS                     2
Mean                17.0999
Median_g            17.1204
Std                0.113532
MedianAbsDev         0.1086
Meanvariance     0.00663934
Eta_e                37.756
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.310193
Period_fit        0.0221484
WMCC_Period        0.309555
WMCC_conf                 1
Name: Blind15A_16_N10_0158_3879, dtype: object
0.3095545
WMCC/LS =  0.997942386831
OLD WMCC period M=3 0.3095545
NEW WMCC period M=1 0.3095545

Blind15A_38_S8_1526_0166
Var_Type                CEP
FLUX_RADIUS           2.546
ELLIPTICITY       0.0900819
FWHM                   4.89
FLAGS                     0
Mean                20.3824
Median_g            20.3238
Std                0.216201
MedianAbsDev         0.2019
Meanvariance      0.0106072
Eta_e               86.0177
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.432791
Period_fit       0.00333307
WMCC_Period        0.432293
WMCC_conf                 1
Name: Blind15A_38_S8_1526_0166, dtype: object
0.432293131261
WMCC/LS =  0.998848589523
OLD WMCC period M=3 0.432293131261
NEW WMCC period M=1 0.432293131261

Blind15A_01_S23_0518_0171
Var_Type                CEP
FLUX_RADIUS          2.5235
ELLIPTICITY       0.0314768
FWHM                   4.49
FLAGS                     0
Mean                16.8026
Median_g             16.799
Std               0.0388769
MedianAbsDev         0.0389
Meanvariance     0.00231374
Eta_e               86.9007
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            0.65408
Period_fit       0.00892861
WMCC_Period        0.034084
WMCC_conf          0.295854
Name: Blind15A_01_S23_0518_0171, dtype: object
0.0340840169923
WMCC/LS =  0.0521098838855
OLD WMCC period M=3 0.0340840169923
NEW WMCC period M=1 0.0340840169923

Blind15A_01_S27_1890_2336
Var_Type                CEP
FLUX_RADIUS           2.637
ELLIPTICITY       0.0614634
FWHM                   4.69
FLAGS                     0
Mean                 17.737
Median_g            17.7281
Std               0.0466665
MedianAbsDev        0.04565
Meanvariance     0.00263103
Eta_e               209.617
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            0.11753
Period_fit       0.00261828
WMCC_Period        0.133367
WMCC_conf          0.999671
Name: Blind15A_01_S27_1890_2336, dtype: object
0.13336731383
WMCC/LS =  1.13475177306
OLD WMCC period M=3 0.13336731383
NEW WMCC period M=1 0.13336731383

Blind15A_05_N29_1320_0672
Var_Type                CEP
FLUX_RADIUS          2.5975
ELLIPTICITY       0.0412263
FWHM                  4.505
FLAGS                     0
Mean                17.6203
Median_g             17.619
Std               0.0419356
MedianAbsDev          0.041
Meanvariance     0.00237995
Eta_e               33.9273
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.888833
Period_fit       0.00786435
WMCC_Period        0.887783
WMCC_conf          0.995765
Name: Blind15A_05_N29_1320_0672, dtype: object
0.887782531915
WMCC/LS =  0.998817966903
OLD WMCC period M=3 0.887782531915
NEW WMCC period M=1 0.887782531915

Blind15A_05_S4_0895_0752
Var_Type                CEP
FLUX_RADIUS          2.5515
ELLIPTICITY       0.0314768
FWHM                  4.505
FLAGS                     0
Mean                15.9131
Median_g            15.8968
Std                0.107906
MedianAbsDev         0.0827
Meanvariance     0.00678099
Eta_e               419.062
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.169732
Period_fit        0.0140259
WMCC_Period        0.169732
WMCC_conf          0.999958
Name: Blind15A_05_S4_0895_0752, dtype: object
0.169731982373
WMCC/LS =  1.0
OLD WMCC period M=3 0.169731982373
NEW WMCC period M=1 0.169731982373

Blind15A_06_S6_0349_1526
Var_Type                CEP
FLUX_RADIUS           2.409
ELLIPTICITY       0.0412272
FWHM                   4.23
FLAGS                     0
Mean                18.2129
Median_g            18.2436
Std                0.195844
MedianAbsDev         0.2108
Meanvariance      0.0107531
Eta_e               181.553
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.345318
Period_fit       0.00783132
WMCC_Period        0.345318
WMCC_conf                 1
Name: Blind15A_06_S6_0349_1526, dtype: object
0.345317544828
WMCC/LS =  1.0
OLD WMCC period M=3 0.345317544828
NEW WMCC period M=1 0.345317544828

Blind15A_07_N10_1524_3310
Var_Type                CEP
FLUX_RADIUS           2.414
ELLIPTICITY       0.0366089
FWHM                   4.12
FLAGS                     0
Mean                 17.919
Median_g            17.9254
Std               0.0535145
MedianAbsDev          0.052
Meanvariance     0.00298646
Eta_e               44.0609
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.878441
Period_fit        0.0126306
WMCC_Period        0.879985
WMCC_conf          0.999938
Name: Blind15A_07_N10_1524_3310, dtype: object
0.87998513181
WMCC/LS =  1.00175746924
OLD WMCC period M=3 0.87998513181
NEW WMCC period M=1 0.87998513181

Blind15A_12_S28_1327_2138
Var_Type               CEP
FLUX_RADIUS          2.456
ELLIPTICITY      0.0662932
FWHM                  4.38
FLAGS                    0
Mean               18.6345
Median_g           18.6367
Std               0.111218
MedianAbsDev         0.083
Meanvariance     0.0059684
Eta_e              225.774
Pred_Var_Type          NaN
Prob_Pred              NaN
PeriodLS          0.286076
Period_fit       0.0113601
WMCC_Period       0.285913
WMCC_conf                1
Name: Blind15A_12_S28_1327_2138, dtype: object
0.285912920617
WMCC/LS =  0.999428897775
OLD WMCC period M=3 0.285912920617
NEW WMCC period M=1 0.285912920617

Blind15A_13_N25_0691_2649
Var_Type                CEP
FLUX_RADIUS            2.24
ELLIPTICITY       0.0430622
FWHM                   4.05
FLAGS                     0
Mean                18.7465
Median_g            18.7436
Std               0.0772372
MedianAbsDev         0.0669
Meanvariance     0.00412008
Eta_e               423.833
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.118353
Period_fit        0.0113704
WMCC_Period        0.118325
WMCC_conf                 1
Name: Blind15A_13_N25_0691_2649, dtype: object
0.118325323091
WMCC/LS =  0.999763649252
OLD WMCC period M=3 0.118325323091
NEW WMCC period M=1 0.118325323091

Blind15A_13_S22_1277_0992
Var_Type                CEP
FLUX_RADIUS           2.267
ELLIPTICITY       0.0503324
FWHM                   4.09
FLAGS                     0
Mean                18.7379
Median_g             18.763
Std               0.0822835
MedianAbsDev          0.063
Meanvariance     0.00439129
Eta_e               47.3228
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.692121
Period_fit        0.0204302
WMCC_Period        0.694361
WMCC_conf                 1
Name: Blind15A_13_S22_1277_0992, dtype: object
0.694361223301
WMCC/LS =  1.00323624595
OLD WMCC period M=3 0.694361223301
NEW WMCC period M=1 0.694361223301

Blind15A_15_N21_0971_2840
Var_Type                CEP
FLUX_RADIUS           2.466
ELLIPTICITY       0.0412272
FWHM                   4.44
FLAGS                     0
Mean                20.2856
Median_g            20.2909
Std               0.0856124
MedianAbsDev         0.0785
Meanvariance     0.00422036
Eta_e               147.024
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            0.38023
Period_fit        0.0172049
WMCC_Period        0.379846
WMCC_conf          0.999964
Name: Blind15A_15_N21_0971_2840, dtype: object
0.379845764795
WMCC/LS =  0.998988366212
OLD WMCC period M=3 0.379845764795
NEW WMCC period M=1 0.379845764795

Blind15A_18_S22_0693_0427
Var_Type               CEP
FLUX_RADIUS         2.3345
ELLIPTICITY      0.0574896
FWHM                 4.305
FLAGS                    0
Mean                19.999
Median_g           19.9873
Std               0.114712
MedianAbsDev       0.10385
Meanvariance     0.0057359
Eta_e               414.43
Pred_Var_Type          NaN
Prob_Pred              NaN
PeriodLS          0.165028
Period_fit       0.0222705
WMCC_Period       0.164937
WMCC_conf         0.999998
Name: Blind15A_18_S22_0693_0427, dtype: object
0.164937057661
WMCC/LS =  0.99945085118
OLD WMCC period M=3 0.164937057661
NEW WMCC period M=1 0.164937057661

Blind15A_19_N22_0427_0718
Var_Type               CEP
FLUX_RADIUS          2.212
ELLIPTICITY      0.0338164
FWHM                  3.89
FLAGS                    0
Mean                 18.03
Median_g           18.0303
Std               0.182945
MedianAbsDev         0.178
Meanvariance     0.0101467
Eta_e               179.46
Pred_Var_Type          NaN
Prob_Pred              NaN
PeriodLS          0.385066
Period_fit       0.0165897
WMCC_Period       0.385066
WMCC_conf                1
Name: Blind15A_19_N22_0427_0718, dtype: object
0.385065707692
WMCC/LS =  1.0
OLD WMCC period M=3 0.385065707692
NEW WMCC period M=1 0.385065707692

Blind15A_22_S27_0428_0159
Var_Type               CEP
FLUX_RADIUS          2.277
ELLIPTICITY      0.0601504
FWHM                  4.02
FLAGS                    0
Mean               18.3307
Median_g           18.3276
Std               0.115819
MedianAbsDev        0.0837
Meanvariance     0.0063183
Eta_e              441.225
Pred_Var_Type          NaN
Prob_Pred              NaN
PeriodLS          0.117325
Period_fit       0.0109663
WMCC_Period       0.117288
WMCC_conf                1
Name: Blind15A_22_S27_0428_0159, dtype: object
0.117288250234
WMCC/LS =  0.999687597619
OLD WMCC period M=3 0.117288250234
NEW WMCC period M=1 0.117288250234

Blind15A_24_N25_1150_3664
Var_Type                CEP
FLUX_RADIUS            2.38
ELLIPTICITY       0.0384615
FWHM                   4.18
FLAGS                     0
Mean                17.1125
Median_g             17.077
Std                0.151516
MedianAbsDev         0.1102
Meanvariance     0.00885411
Eta_e               430.412
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.150931
Period_fit        0.0112148
WMCC_Period        0.150961
WMCC_conf                 1
Name: Blind15A_24_N25_1150_3664, dtype: object
0.15096089807
WMCC/LS =  1.00020104544
OLD WMCC period M=3 0.15096089807
NEW WMCC period M=1 0.15096089807

Blind15A_24_S25_0750_1372
Var_Type                CEP
FLUX_RADIUS            2.52
ELLIPTICITY       0.0662932
FWHM                    4.8
FLAGS                     0
Mean                17.7686
Median_g            17.7711
Std               0.0365419
MedianAbsDev         0.0237
Meanvariance     0.00205654
Eta_e               23.2373
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            1.83141
Period_fit        0.0287977
WMCC_Period         1.82696
WMCC_conf          0.819171
Name: Blind15A_24_S25_0750_1372, dtype: object
1.82695743796
WMCC/LS =  0.997566909976
OLD WMCC period M=3 1.82695743796
NEW WMCC period M=1 1.82695743796

Blind15A_28_N1_1118_2791
Var_Type               CEP
FLUX_RADIUS          2.498
ELLIPTICITY      0.0507812
FWHM                  4.51
FLAGS                    0
Mean               16.0208
Median_g           16.0276
Std              0.0248273
MedianAbsDev       0.01675
Meanvariance     0.0015497
Eta_e              13.1569
Pred_Var_Type          NaN
Prob_Pred              NaN
PeriodLS           6.82623
Period_fit        0.018374
WMCC_Period        6.95264
WMCC_conf         0.994576
Name: Blind15A_28_N1_1118_2791, dtype: object
6.95263805555
WMCC/LS =  1.01851851852
OLD WMCC period M=3 6.95263805555
NEW WMCC period M=1 6.95263805555

Blind15A_29_N15_1943_1079
Var_Type                CEP
FLUX_RADIUS           2.325
ELLIPTICITY       0.0494297
FWHM                    4.1
FLAGS                     0
Mean                17.6647
Median_g            17.6662
Std               0.0504103
MedianAbsDev        0.04655
Meanvariance     0.00285373
Eta_e               46.6746
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.921333
Period_fit        0.0165933
WMCC_Period        0.921898
WMCC_conf          0.998779
Name: Blind15A_29_N15_1943_1079, dtype: object
0.921898379374
WMCC/LS =  1.00061387354
OLD WMCC period M=3 0.921898379374
NEW WMCC period M=1 0.921898379374

Blind15A_32_N5_1662_3867
Var_Type                CEP
FLUX_RADIUS           2.306
ELLIPTICITY       0.0338164
FWHM                   3.88
FLAGS                     0
Mean                16.8695
Median_g            16.8765
Std               0.0344025
MedianAbsDev         0.0346
Meanvariance     0.00203933
Eta_e               134.949
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.386061
Period_fit        0.0184197
WMCC_Period        0.386259
WMCC_conf          0.998754
Name: Blind15A_32_N5_1662_3867, dtype: object
0.386259192901
WMCC/LS =  1.00051440329
OLD WMCC period M=3 0.386259192901
NEW WMCC period M=1 0.386259192901

Blind15A_33_N22_1314_3152
Var_Type                CEP
FLUX_RADIUS            2.33
ELLIPTICITY       0.0448902
FWHM                   4.03
FLAGS                     0
Mean                18.3314
Median_g            18.3306
Std                0.043296
MedianAbsDev         0.0364
Meanvariance     0.00236185
Eta_e               25.7761
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            1.41677
Period_fit       0.00928491
WMCC_Period         1.41811
WMCC_conf          0.999935
Name: Blind15A_33_N22_1314_3152, dtype: object
1.41810713314
WMCC/LS =  1.00094428706
OLD WMCC period M=3 1.41810713314
NEW WMCC period M=1 1.41810713314

Blind15A_36_N31_1448_3687
Var_Type                CEP
FLUX_RADIUS           2.453
ELLIPTICITY       0.0272374
FWHM                   4.05
FLAGS                     0
Mean                 18.124
Median_g            18.1455
Std                 0.04281
MedianAbsDev         0.0239
Meanvariance     0.00236206
Eta_e               14.1743
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            1.80937
Period_fit        0.0173002
WMCC_Period          1.8072
WMCC_conf          0.998728
Name: Blind15A_36_N31_1448_3687, dtype: object
1.80719516246
WMCC/LS =  0.998796630571
OLD WMCC period M=3 1.80719516246
NEW WMCC period M=1 1.80719516246

Blind15A_40_N22_0886_3892
Var_Type               CEP
FLUX_RADIUS           2.26
ELLIPTICITY       0.032882
FWHM                   3.9
FLAGS                    0
Mean               18.8843
Median_g            18.892
Std               0.043007
MedianAbsDev        0.0353
Meanvariance     0.0022774
Eta_e              36.3519
Pred_Var_Type          NaN
Prob_Pred              NaN
PeriodLS           0.77015
Period_fit       0.0178808
WMCC_Period       0.768967
WMCC_conf         0.997939
Name: Blind15A_40_N22_0886_3892, dtype: object
0.768966847926
WMCC/LS =  0.99846390169
OLD WMCC period M=3 0.768966847926
NEW WMCC period M=1 0.768966847926

Blind15A_41_N28_0904_3534
Var_Type                CEP
FLUX_RADIUS          2.2705
ELLIPTICITY       0.0471651
FWHM                  3.865
FLAGS                     0
Mean                18.7164
Median_g             18.703
Std                0.100495
MedianAbsDev         0.0745
Meanvariance     0.00536935
Eta_e               370.921
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.130138
Period_fit        0.0106367
WMCC_Period        0.130161
WMCC_conf                 1
Name: Blind15A_41_N28_0904_3534, dtype: object
0.130160677587
WMCC/LS =  1.00017334026
OLD WMCC period M=3 0.130160677587
NEW WMCC period M=1 0.130160677587

Blind15A_42_N3_1613_3563
Var_Type                CEP
FLUX_RADIUS           2.297
ELLIPTICITY       0.0366089
FWHM                      4
FLAGS                     0
Mean                17.8948
Median_g            17.8752
Std               0.0994134
MedianAbsDev         0.0805
Meanvariance     0.00555544
Eta_e               172.855
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.279664
Period_fit       0.00787679
WMCC_Period        0.279508
WMCC_conf                 1
Name: Blind15A_42_N3_1613_3563, dtype: object
0.27950825014
WMCC/LS =  0.999441652709
OLD WMCC period M=3 0.27950825014
NEW WMCC period M=1 0.27950825014

Blind15A_42_S13_0990_0492
Var_Type                CEP
FLUX_RADIUS           2.374
ELLIPTICITY       0.0338164
FWHM                   4.15
FLAGS                     2
Mean                17.1858
Median_g            17.1628
Std                0.143831
MedianAbsDev         0.1094
Meanvariance     0.00836917
Eta_e               293.108
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.151391
Period_fit       0.00422957
WMCC_Period        0.046168
WMCC_conf           0.99762
Name: Blind15A_42_S13_0990_0492, dtype: object
0.0461679679056
WMCC/LS =  0.304958652279
OLD WMCC period M=3 0.0461679679056
NEW WMCC period M=1 0.0461679679056

Blind15A_42_S29_0059_2876
Var_Type                CEP
FLUX_RADIUS          2.6795
ELLIPTICITY       0.0990633
FWHM                   5.11
FLAGS                     0
Mean                17.5469
Median_g             17.537
Std                0.101424
MedianAbsDev         0.0943
Meanvariance     0.00578017
Eta_e               218.101
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.303394
Period_fit        0.0062011
WMCC_Period        0.303394
WMCC_conf                 1
Name: Blind15A_42_S29_0059_2876, dtype: object
0.303393500606
WMCC/LS =  1.0
OLD WMCC period M=3 0.303393500606
NEW WMCC period M=1 0.303393500606

Blind15A_43_N7_1350_1603
Var_Type                CEP
FLUX_RADIUS           2.399
ELLIPTICITY       0.0430622
FWHM                   4.21
FLAGS                     0
Mean                 17.393
Median_g            17.3746
Std                0.126928
MedianAbsDev         0.1033
Meanvariance     0.00729762
Eta_e               364.805
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            0.12599
Period_fit       0.00799119
WMCC_Period        0.126032
WMCC_conf                 1
Name: Blind15A_43_N7_1350_1603, dtype: object
0.126032191843
WMCC/LS =  1.00033568312
OLD WMCC period M=3 0.126032191843
NEW WMCC period M=1 0.126032191843

Blind15A_43_S8_1055_1696
Var_Type                CEP
FLUX_RADIUS           2.469
ELLIPTICITY       0.0592662
FWHM                   4.56
FLAGS                     0
Mean                17.9162
Median_g            17.8891
Std               0.0903185
MedianAbsDev         0.0768
Meanvariance     0.00504117
Eta_e               3.04587
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            2.08583
Period_fit       0.00332081
WMCC_Period         2.08583
WMCC_conf                 1
Name: Blind15A_43_S8_1055_1696, dtype: object
2.085832775
WMCC/LS =  1.0
OLD WMCC period M=3 2.085832775
NEW WMCC period M=1 2.085832775

Blind15A_47_N2_1273_0173
Var_Type                CEP
FLUX_RADIUS          2.4965
ELLIPTICITY       0.0605916
FWHM                  4.535
FLAGS                     0
Mean                17.9946
Median_g             17.975
Std               0.0854766
MedianAbsDev         0.0684
Meanvariance     0.00475013
Eta_e                423.62
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.168435
Period_fit       0.00720118
WMCC_Period        0.168378
WMCC_conf          0.999996
Name: Blind15A_47_N2_1273_0173, dtype: object
0.168378426119
WMCC/LS =  0.999663412991
OLD WMCC period M=3 0.168378426119
NEW WMCC period M=1 0.168378426119

Blind15A_47_S17_0964_2825
Var_Type                CEP
FLUX_RADIUS           2.414
ELLIPTICITY       0.0430622
FWHM                   4.15
FLAGS                     0
Mean                16.4604
Median_g            16.4647
Std               0.0528089
MedianAbsDev          0.034
Meanvariance     0.00320823
Eta_e                520.18
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS           0.125167
Period_fit         0.028807
WMCC_Period        0.125126
WMCC_conf          0.999682
Name: Blind15A_47_S17_0964_2825, dtype: object
0.125125638819
WMCC/LS =  0.999666499912
OLD WMCC period M=3 0.125125638819
NEW WMCC period M=1 0.125125638819

Blind15A_48_N13_1900_3489
Var_Type                CEP
FLUX_RADIUS           2.381
ELLIPTICITY       0.0507829
FWHM                  4.315
FLAGS                     0
Mean                16.3953
Median_g            16.3764
Std                0.138943
MedianAbsDev         0.1434
Meanvariance     0.00847455
Eta_e               1.08456
Pred_Var_Type           NaN
Prob_Pred               NaN
PeriodLS            6.82162
Period_fit         0.016905
WMCC_Period         8.20086
WMCC_conf                 1
Name: Blind15A_48_N13_1900_3489, dtype: object
8.20085606557
WMCC/LS =  1.20218579235
OLD WMCC period M=3 8.20085606557
NEW WMCC period M=1 8.20085606557

plt.hist(table_15.query('Var_Type=="CEP"').Amplitude, bins=15, alpha=.5)
plt.hist(table_15.query('Var_Type=="RRLYR"').Amplitude, bins=25, alpha=.5)
plt.show()

cep_ = table_15.query('Var_Type == "CEP"')
rrl_ = table_15.query('Var_Type == "RRLYR"')
ebs_ = table_15.query('Var_Type == "EB"')
plt.plot(table_15.Color, table_15.Mean, '.b')
plt.plot(cep_.Color, cep_.Mean, '*r')
plt.plot(rrl_.Color, rrl_.Mean, '*g')
#plt.plot(ebs_.Color, ebs_.Mean, '*c')
plt.xlim(-3,3)
plt.ylim(15,25)
plt.show()

from astropy.coordinates import SkyCoord from astropy import units as u c = SkyCoord(ra=table_15.loc['Blind14A_29_N31_0314_2793','RA'], dec=table_15.loc['Blind14A_29_N31_0314_2793','DEC'], unit=(u.degree, u.degree)) print c.ra.hms print c.dec.radian

table_15.loc[table_15.query('Std<0.01 and Meanvariance<0.01 and MedianAbsDev<0.01').sample(95).index, 'Var_Type'] = 'NV'

table_15.info()

<class 'pandas.core.frame.DataFrame'>
Index: 965142 entries, Blind14A_01_N10_0020_1882 to Blind14A_40_S9_2013_0916
Data columns (total 79 columns):
DEC                                965142 non-null float64
ELLIPTICITY                        965142 non-null float64
FLAGS                              965142 non-null int64
FLUX_RADIUS                        965142 non-null float64
FWHM                               965142 non-null float64
KRON_RADIUS                        965142 non-null float64
Median_err_g                       965142 non-null float64
Median_g                           965142 non-null float64
Occu_g                             965142 non-null int64
RA                                 965142 non-null float64
STD_g                              965142 non-null float64
X                                  965142 non-null float64
Y                                  965142 non-null float64
Amplitude                          965142 non-null float64
AndersonDarling                    965142 non-null float64
Autocor_length                     965142 non-null float64
Beyond1Std                         965142 non-null float64
CAR_mean                           965142 non-null float64
CAR_sigma                          965142 non-null float64
CAR_tau                            965142 non-null float64
Con                                965142 non-null float64
Eta_e                              965142 non-null float64
FluxPercentileRatioMid20           965142 non-null float64
FluxPercentileRatioMid35           965142 non-null float64
FluxPercentileRatioMid50           965142 non-null float64
FluxPercentileRatioMid65           965142 non-null float64
FluxPercentileRatioMid80           965142 non-null float64
Freq1_harmonics_amplitude_0        965142 non-null float64
Freq1_harmonics_amplitude_1        965142 non-null float64
Freq1_harmonics_amplitude_2        965142 non-null float64
Freq1_harmonics_amplitude_3        965142 non-null float64
Freq1_harmonics_rel_phase_0        965142 non-null float64
Freq1_harmonics_rel_phase_1        965142 non-null float64
Freq1_harmonics_rel_phase_2        965142 non-null float64
Freq1_harmonics_rel_phase_3        965142 non-null float64
Freq2_harmonics_amplitude_0        965142 non-null float64
Freq2_harmonics_amplitude_1        965142 non-null float64
Freq2_harmonics_amplitude_2        965142 non-null float64
Freq2_harmonics_amplitude_3        965142 non-null float64
Freq2_harmonics_rel_phase_0        965142 non-null float64
Freq2_harmonics_rel_phase_1        965142 non-null float64
Freq2_harmonics_rel_phase_2        965142 non-null float64
Freq2_harmonics_rel_phase_3        965142 non-null float64
Freq3_harmonics_amplitude_0        965142 non-null float64
Freq3_harmonics_amplitude_1        965142 non-null float64
Freq3_harmonics_amplitude_2        965142 non-null float64
Freq3_harmonics_amplitude_3        965142 non-null float64
Freq3_harmonics_rel_phase_0        965142 non-null float64
Freq3_harmonics_rel_phase_1        965142 non-null float64
Freq3_harmonics_rel_phase_2        965142 non-null float64
Freq3_harmonics_rel_phase_3        965142 non-null float64
Gskew                              965141 non-null float64
LinearTrend                        965142 non-null float64
MaxSlope                           965142 non-null float64
Mean                               965142 non-null float64
Meanvariance                       965142 non-null float64
MedianAbsDev                       965142 non-null float64
MedianBRP                          965142 non-null float64
PairSlopeTrend                     965142 non-null float64
PercentAmplitude                   965142 non-null float64
PercentDifferenceFluxPercentile    965142 non-null float64
PeriodLS                           965142 non-null float64
Period_fit                         965142 non-null float64
Psi_CS                             965142 non-null float64
Psi_eta                            965142 non-null float64
Q31                                965142 non-null float64
Rcs                                965142 non-null float64
Skew                               965142 non-null float64
SlottedA_length                    965142 non-null float64
SmallKurtosis                      965142 non-null float64
Std                                965142 non-null float64
StetsonK                           965142 non-null float64
StetsonK_AC                        965142 non-null float64
WMCC_Period                        965142 non-null float64
WMCC_conf                          965142 non-null float64
spCl                               8656 non-null object
subClass                           5506 non-null object
Var_Type                           1791 non-null object
Var_subType                        19 non-null object
dtypes: float64(73), int64(2), object(4)
memory usage: 589.1+ MB

#table_15.to_csv('%s/tables/__Blind15A_tables+feat_filter_type_sub_sp.csv'
#             % (mainpath))